from Eda import Eda
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from math import log
from xgboost import XGBRegressor
from sklearn.preprocessing import OneHotEncoder
from sklearn import preprocessing
from scipy import stats
'''
声明eda对象
'''
eda = Eda()  
'''
读入训练集与测试集合
'''
trainSet = pd.read_csv('train.csv')
testSet = pd.read_csv('test.csv')
'''
将测试集合和训练集合合并
'''
allSet = eda.allSet(train=trainSet,test=testSet,labelName='SalePrice')
#print(allSet.head())
'''
获取不含有NA的列
'''
nnaCols = eda.colWithNNa(allSet)
'''
获取有NA值的列
'''
naCols = eda.nameOfNASort(allSet)
'''
PoolQC          2909.0
MiscFeature     2814.0
Alley           2721.0
Fence           2348.0
SalePrice       1459.0
FireplaceQu     1420.0
LotFrontage      486.0
GarageQual       159.0
GarageYrBlt      159.0
GarageFinish     159.0
GarageCond       159.0
GarageType       157.0
BsmtExposure      82.0
BsmtCond          82.0
BsmtQual          81.0
BsmtFinType2      80.0
BsmtFinType1      79.0
MasVnrType        24.0
MasVnrArea        23.0
MSZoning           4.0
Utilities          2.0
Functional         2.0
BsmtFullBath       2.0
BsmtHalfBath       2.0
GarageArea         1.0
BsmtFinSF2         1.0
Exterior1st        1.0
TotalBsmtSF        1.0
GarageCars         1.0
BsmtUnfSF          1.0
Electrical         1.0
BsmtFinSF1         1.0
KitchenQual        1.0
SaleType           1.0
Exterior2nd        1.0
填充这些NA值，填充的时候结合所有的columns，不仅仅是具有NA的columns进行，需要对columns根据业务分组
'''
'''
首先填充NA最多的PoolQC，有关Pool的变量有3个，分别是PoolQC、PoolArea以及OverallQual，
因为PoolArea和OverallQual没有空值，如果PoolArea>0，则选取相应idx的OverallQual值填入
PoolQC
'''
#idx = eda.idxOfNa(allSet,'PoolQC',eda.idxOfGreatThan0(allSet['PoolArea']))
allSet.loc[[2420, 2503, 2599],['PoolQC']]=['Fa','TA','Fa']
allSet['PoolQC']=allSet['PoolQC'].fillna('None')
'''
经过分析,PoolQC具有order关系，需要转换
'''
allSet['PoolQC'] = eda.seConvNum(allSet['PoolQC'],['None','Po','Fa','Ta','Gd','Ex','FA','TA'],[0,1,2,3,4,5,2,3])
'''
MiscFeature,Alley,Fence填充None
'''
allSet['MiscFeature'] = allSet['MiscFeature'].fillna('None')
allSet['Alley'] = allSet['Alley'].fillna('None')
allSet['Fence'] = allSet['Fence'].fillna('Fence')
'''
FireplaceQu填充NA为None，并且是ordinal
'''
allSet['FireplaceQu'] = allSet['FireplaceQu'].fillna('None')
allSet['FireplaceQu'] = eda.seConvNum(allSet['FireplaceQu'],['None','Po','Fa','TA','Gd','Ex'],[1,0,2,3,4,5])
'''
LotFrontage其中LotFrontage表示庭院距离街道的距离，有486个NA值。这个值可以使用Neighborhood的LotFrontage平均值表示。
'''
allSet=eda.fillNAWithOtherGroupMean(allSet,'LotFrontage','Neighborhood')
'''
Garage车库相关特征（GarageYrBlt、GarageType、GarageFinish、GarageCond、GarageQual、GarageArea、GarageCars）
'''
'''
GarageYrBlt--车库修建年代，有159个NA。车库修建年代和房屋的修建年代在大多数情况下是相等的， 因此可以用YearBuilt（房屋修建年代）代替车库修建年代。
'''
allSet=eda.fillNAWithOtherGroupMean(allSet,'GarageYrBlt','YearBuilt')
'''
根据Garage的综合参数，确定多少House没有Garage
'''
len(allSet[allSet['GarageType'].isna()&allSet['GarageFinish'].isna()&allSet['GarageCond'].isna()&allSet['GarageQual'].isna()&(allSet['GarageArea']==0.0)&(allSet['GarageCars']==0.0)])
'''
157个，说明还有两个GarageType有值但是GarageFinish、GarageCond、GarageQual没有值的
经过分析，可知index为2576的没有Garage，共158个house没有Garage，2126使用最普遍的填充
'''
eda.fillNAWithCommon(allSet,['GarageFinish','GarageCond','GarageQual'],[2126])
allSet.at[2576,'GarageType']=np.nan
allSet.at[2576,'GarageArea']=0
allSet.at[2576,'GarageCars']=0
'''
剩下的NA填充None
'''
allSet['GarageType'] = allSet['GarageType'].fillna('None')
allSet['GarageFinish'] = allSet['GarageFinish'].fillna('None')
allSet['GarageCond'] = allSet['GarageCond'].fillna('None')
allSet['GarageQual'] = allSet['GarageQual'].fillna('None')
'''
通过分析，可知GarageCond、GarageFinish、GarageQual有Ordinal关系
'''
allSet['GarageFinish']=eda.seConvNum(allSet['GarageFinish'],['None','Unf','RFn','Fin'],[0,1,2,3])
allSet['GarageCond']=eda.seConvNum(allSet['GarageCond'],['None','Po','Fa','TA','Gd','Ex'],[0,1,2,3,4,5])
allSet['GarageQual']=eda.seConvNum(allSet['GarageQual'],['None','Po','Fa','TA','Gd','Ex'],[0,1,2,3,4,5])
'''
Basement Variables(有关地下室的特征)，Basement相关的变量一共11个，分别是BsmtQual、BsmtCond、BsmtExposure、BsmtFinType1、BsmtFinType2、BsmtFullBath、BsmtHalfBath、BsmtFinSF1、BsmtFinSF2、BsmtUnfSF、TotalBsmtSF
其中，BsmtQual、BsmtCond、BsmtExposure、BsmtFinType1、BsmtFinType2有79~82个NA。而BsmtFullBath、BsmtHalfBath、BsmtFinSF1、BsmtFinSF2、BsmtUnfSF、TotalBsmtSF都只有1~2个NA。
BsmtQual、BsmtCond、BsmtExposure、BsmtFinType1、BsmtFinType2这几个特征中，所有特征都为NA的数目为：
'''
len(allSet[allSet['BsmtQual'].isna()&allSet['BsmtCond'].isna()&allSet['BsmtExposure'].isna()&allSet['BsmtFinType1'].isna()&allSet['BsmtFinType2'].isna()])
'''
共79个，其中BsmtFinType1就有79个NA，因此可以初步判断，有79个House没有Basement。查看BsmtQual、BsmtCond、BsmtExposure、BsmtFinType2为NA而BsmtFinType1不为NA的数据，
allSet[(-allSet['BsmtFinType1'].isna())&(allSet['BsmtQual'].isna()|allSet['BsmtCond'].isna()|allSet['BsmtExposure'].isna()|allSet['BsmtFinType2'].isna())]
使用common填充
'''
eda.fillNAWithCommon(allSet,['BsmtQual'],[2217,2218])
eda.fillNAWithCommon(allSet,['BsmtExposure'],[948,1487,2348])
eda.fillNAWithCommon(allSet,['BsmtCond'],[2040,2185,2524])
eda.fillNAWithCommon(allSet,['BsmtFinType2'],[332])
#将BsmtQual BsmtCond BsmtExposure BsmtFinType1 BsmtFinType2 的NA
allSet['BsmtQual'] = allSet['BsmtQual'].fillna('None')
allSet['BsmtCond'] = allSet['BsmtCond'].fillna('None')
allSet['BsmtExposure'] = allSet['BsmtExposure'].fillna('None')
allSet['BsmtFinType1'] = allSet['BsmtFinType1'].fillna('None')
allSet['BsmtFinType2'] = allSet['BsmtFinType2'].fillna('None')
#将BsmtFullBath、BsmtHalfBath、BsmtFinSF1、BsmtFinSF2、BsmtUnfSF、TotalBsmtSF的NA填充为0
allSet['BsmtFullBath'] = allSet['BsmtFullBath'].fillna(0)
allSet['BsmtHalfBath'] = allSet['BsmtHalfBath'].fillna(0)
allSet['BsmtFinSF1'] = allSet['BsmtFinSF1'].fillna(0)
allSet['BsmtFinSF2'] = allSet['BsmtFinSF2'].fillna(0)
allSet['BsmtUnfSF'] = allSet['BsmtUnfSF'].fillna(0)
allSet['TotalBsmtSF'] = allSet['TotalBsmtSF'].fillna(0)
'''
从BsmtQual、BsmtCond、BsmtExposure、BsmtFinType1、BsmtFinType2的定义可以看出，他们都是ordinal变量，因此需要变成整形。
'''
allSet['BsmtQual']=eda.seConvNum(allSet['BsmtQual'],['None','Po','Fa','TA','Gd','Ex'],[0,1,2,3,4,5])
allSet['BsmtCond']=eda.seConvNum(allSet['BsmtCond'],['None','Po','Fa','TA','Gd','Ex'],[0,1,2,3,4,5])
allSet['BsmtExposure']=eda.seConvNum(allSet['BsmtExposure'],['None','No','Mn','Av','Gd'],[0,1,2,3,4])
allSet['BsmtFinType1']=eda.seConvNum(allSet['BsmtFinType1'],['None','Unf','LwQ','Rec','BLQ','ALQ','GLQ'],[0,1,2,3,4,5,6])
allSet['BsmtFinType2']=eda.seConvNum(allSet['BsmtFinType2'],['None','Unf','LwQ','Rec','BLQ','ALQ','GLQ'],[0,1,2,3,4,5,6])
'''
masonry（石材）相关变量。石材相关变量有2个，MasVnrType MasVnrArea。其中MasVnrType有24个NA，MasVnrArea有23个NA。经过分析填充
'''
eda.fillNAWithCommon(allSet,['MasVnrType'],[2610])
allSet['MasVnrType'] = allSet['MasVnrType'].fillna('None')
allSet['MasVnrArea'] = allSet['MasVnrArea'].fillna(0)
allSet['MasVnrType']=eda.seConvNum(allSet['MasVnrType'],['None','BrkCmn','BrkFace','Stone'],[0,0,1,2])
'''
MSZoning 有4个NA值，根据定义，可以使用最common的值进行填充。
'''
eda.fillNAWithCommon(allSet,['MSZoning'],[1915,2216,2250,2904])
'''
找出Kitchen quality中的NA,Kitchen quality只有一个NA，使用common填充
'''
eda.fillNAWithCommon(allSet,['KitchenQual'],eda.ofNa(allSet,'KitchenQual').values.tolist())
allSet['KitchenQual']=eda.seConvNum(allSet['KitchenQual'],['Po','Fa','TA','Gd','Ex'],[0,1,2,3,4])
'''
Utilities 和Kitchen quality做同样的处理
'''
eda.fillNAWithCommon(allSet,['Utilities'],eda.ofNa(allSet,'Utilities').values.tolist())
allSet['Utilities']=eda.seConvNum(allSet['Utilities'],['ELO','NoSeWa','NoSewr','AllPub'],[0,1,2,3])
'''
Home functionality 和Kitchen quality做同样的处理
'''
eda.fillNAWithCommon(allSet,['Functional'],eda.ofNa(allSet,'Functional').values.tolist())
allSet['Functional']=eda.seConvNum(allSet['Functional'],['Sal','Sev','Maj2','Maj1','Mod','Min2','Min1','Typ'],[0,1,2,3,4,5,6,7])
'''
Electrical: Electrical system SaleType: Type of sale 都使用最common的填充即可
Exterior1st: Exterior covering on house 
Exterior2nd: Exterior covering on house (if more than one material)
'''
eda.fillNAWithCommon(allSet,['Electrical'],eda.ofNa(allSet,'Electrical').values.tolist())
eda.fillNAWithCommon(allSet,['SaleType'],eda.ofNa(allSet,'SaleType').values.tolist())
eda.fillNAWithCommon(allSet,['Exterior1st'],eda.ofNa(allSet,'Exterior1st').values.tolist())
eda.fillNAWithCommon(allSet,['Exterior2nd'],eda.ofNa(allSet,'Exterior2nd').values.tolist())
'''
ExterQual ExterCond 没有NA，但是是Ordinal类型，需要转换
'''
allSet['ExterQual']=eda.seConvNum(allSet['ExterQual'],['Po','Fa','TA','Gd','Ex'],[0,1,2,3,4])
allSet['ExterCond']=eda.seConvNum(allSet['ExterCond'],['Po','Fa','TA','Gd','Ex'],[0,1,2,3,4])
'''
----------------------------------------------------------------------------------------------------------------------------------
处理allSet中不含NA的列
'''
'''
HeatingQC 具有Ordinal关系，因此需要做转换
'''
allSet['HeatingQC']=eda.seConvNum(allSet['HeatingQC'],['Po','Fa','TA','Gd','Ex'],[0,1,2,3,4])
'''
CentralAir
   N    No
   Y    Yes
转成Y=1 N=0
'''
allSet['CentralAir']=eda.seConvNum(allSet['CentralAir'],['N','Y'],[0,1])
'''
LandSlope
'Sev'=0, 'Mod'=1, 'Gtl'=2
'''
allSet['LandSlope']=eda.seConvNum(allSet['LandSlope'],['Sev','Mod','Gtl'],[0,1,2])
'''
Street
   Grvl Gravel  
   Pave Paved
'''
allSet['Street']=eda.seConvNum(allSet['Street'],['Grvl','Pave'],[0,1])
'''
PavedDrive: Paved driveway
   Y    Paved 
   P    Partial Pavement
   N    Dirt/Gravel
'''
allSet['PavedDrive']=eda.seConvNum(allSet['PavedDrive'],['Y','P','N'],[0,1,2])
'''
MSSubClass变量，虽然表面看是数值型，但是实际上是类别型
'''
allSet['MSSubClass'].astype('object')
'''
对数值型变量的研究（相关系数），需要先看数值型变量的相关系数
'''
'''
找出数值变量对应的的dataframe,需要从trainSet中找，因为testSet中没有SalePrice值,并且去除'Id','MSSubClass','YrSold','MoSold'几个变量
'''
#trainSet = allSet.where(allSet['Id']<1461)
trainSet = allSet.loc[0:1459]
testSet = allSet.loc[1460:]
numericCols = eda.numericSet(allSet).drop(columns=['Id','MSSubClass','YrSold','MoSold']).columns.tolist()
numericSet = trainSet[numericCols].astype('float')
corr = eda.corr(numericSet,'SalePrice')
print(corr['SalePrice'])
'''
将枚举型的进行编号
'''
#使用H2ORandomForest评估特征重要性（因为可以处理非数值型输入）
'''
trainSetCnv = eda.likelyhoodEncodingFrame(trainSet,'SalePrice')
model = XGBRegressor()
X = trainSetCnv.drop(['SalePrice'],axis=1)
y= trainSetCnv['SalePrice']
model.fit(X.values,y.values)
importances = {}
i = 0
for col in X.columns:
    importances[col]=model.feature_importances_[i]
    i=i+1
importancesDF = pd.DataFrame(sorted(importances.items(),key=lambda d:d[1],reverse=True))
print(importancesDF)
'''
'''
通过随机森林的排名，可以进一步看到中top20中数值的分布,这20个变量为
GrLivArea
Neighborhood
OverallQual
MsSubClass
TotalBsmtSF
BsmtFlnSF1
1stFlrSF
LotArea
OverallCond
BsmtFlnType1
GarageType
2ndFlrSF
GarageCars
ExterQual        
GarageArea       
FullBath         
FireplaceQu      
Exterior2nd        
GarageFinish
CentralAir
先查看一些与面积相关的
GrLivArea
GarageArea
TotalBsmtSF
X1stFlrSF
'''
'''
查看面积相关的变量GrLivArea 1stFlrSF 2ndFlrSF TotalBsmtSF
'''
fig,axes = plt.subplots(2,2)
sns.distplot(trainSet['GrLivArea'],axlabel="Square feet living area",ax=axes[0,0])
sns.distplot(trainSet['1stFlrSF'],axlabel="Square feet first floor",ax=axes[0,1])
sns.distplot(trainSet['2ndFlrSF'],axlabel="Square feet second floor",ax=axes[1,0])
sns.distplot(trainSet['TotalBsmtSF'],axlabel="Square feet basement",ax=axes[1,1])
plt.show()
'''
从列定义上，似乎可以看出1stFlrSF 2ndFlrSF LowQualFinSF 与GrLivArea 有较强的相关关系
'''
totalarea = trainSet['1stFlrSF']+trainSet['2ndFlrSF']+trainSet['LowQualFinSF']
totalareaCorr = totalarea.corr(trainSet['GrLivArea'])
'''
print(totalareaCorr)
基本上等于1，因此有非常强的线形相关关系
'''
#sns.barplot(x=1,y=0,data=importancesDF)
# plot feature importance
#plot_importance(model)
#plt.show()
'''
使用直方图查看OverallQual ExterQual BsmtQual KitchenQual GarageQual FireplaceQu PoolQC
'''
fig,axes = plt.subplots(3,3)
sns.countplot(trainSet['OverallQual'],ax=axes[0,0])
sns.countplot(trainSet['ExterQual'],ax=axes[0,1])
sns.countplot(trainSet['BsmtQual'],ax=axes[0,2])
sns.countplot(trainSet['KitchenQual'],ax=axes[1,0])
sns.countplot(trainSet['GarageQual'],ax=axes[1,1])
sns.countplot(trainSet['FireplaceQu'],ax=axes[1,2])
sns.countplot(trainSet['PoolQC'],ax=axes[2,0])
plt.show()
'''
从直方图可以看出，PoolQC大多数样本都集中在‘0’，区分度很低，需要去掉
GarageQual样本区分度很低，需要去掉
'''
'''
使用box图观察Neighborhood，查看这个变量与SalePrice的关系
'''
sns.boxplot(x=trainSet['Neighborhood'],y=trainSet['SalePrice'])
plt.show()
'''
展示不同类别的数量
'''
sns.countplot(trainSet['Neighborhood'])
plt.show()
'''
使用box图观察MSSubClass
'''
sns.boxplot(trainSet['MSSubClass'],trainSet['SalePrice'],ax=axes[0,0])
plt.show()
'''
展示不同类别的数量
'''
sns.countplot(trainSet['MSSubClass'],ax=axes[1,0])
plt.show()
'''
分析garage相关变量 GarageYrBlt GarageCars GarageArea GarageCond GarageType GarageQual GarageFinish
经过画图可以得知，GarageQual，GarageCond主要的数据集中在一个取值上
'''
fig,axes = plt.subplots(3,3)
sns.countplot(trainSet['GarageYrBlt'],ax=axes[0,0])
sns.countplot(trainSet['GarageCars'],ax=axes[0,1])
sns.distplot(trainSet['GarageArea'],ax=axes[0,2])
sns.countplot(trainSet['GarageCond'],ax=axes[1,0])
sns.countplot(trainSet['GarageType'],ax=axes[1,1])
sns.countplot(trainSet['GarageQual'],ax=axes[1,2])
sns.countplot(trainSet['GarageFinish'],ax=axes[2,0])
plt.show()
'''
同样的方法可以分析basement相关变量
从图中可以看出BsmtFinSF2 大多数房屋为0
BsmtFinType2集中在一个取值上
'''
fig,axes = plt.subplots(3,3)
sns.distplot(trainSet['BsmtFinSF1'],ax=axes[0,0])
sns.distplot(trainSet['BsmtFinSF2'],ax=axes[0,1])
sns.distplot(trainSet['BsmtUnfSF'],ax=axes[0,2])
sns.countplot(trainSet['BsmtFinType1'],ax=axes[1,0])
sns.countplot(trainSet['BsmtFinType2'],ax=axes[1,1])
sns.countplot(trainSet['BsmtQual'],ax=axes[1,2])
sns.countplot(trainSet['BsmtCond'],ax=axes[2,0])
sns.countplot(trainSet['BsmtExposure'],ax=axes[2,1])
plt.show()
'''
从定义上看，Total Basement Surface in square feet (TotalBsmtSF) 应该是BsmtFinSF1 BsmtFinSF2和BsmtUnfSF的和
计算相关系数，为1.0，证明猜想完全正确
'''
totalarea = trainSet['BsmtFinSF1']+trainSet['BsmtFinSF2']+trainSet['BsmtUnfSF']
totalareaCorr = totalarea.corr(trainSet['TotalBsmtSF'])
print(totalareaCorr)
'''
查看BathRoom相关的变量，它不属于importance feature
但是从定义上理解，如果
TotBathrooms = FullBath + (HalfBath*0.5) + BsmtFullBath + (BsmtHalfBath*0.5)
再计算它和SalePrice的相关系数，检验相关系数为0.63，比较高，可以加入
'''
totalBathRooms =  trainSet['FullBath']+0.5*trainSet['HalfBath']+trainSet['BsmtFullBath']+0.5*trainSet['BsmtHalfBath']
testTotalBathRooms =  testSet['FullBath']+0.5*testSet['HalfBath']+testSet['BsmtFullBath']+0.5*testSet['BsmtHalfBath']
totalareaCorr = totalBathRooms.corr(trainSet['SalePrice'])
print(totalareaCorr)
if totalareaCorr>0.5 or totalareaCorr<-0.5:
   trainSet['TotalBathRooms']=totalBathRooms
   testSet['TotalBathRooms']=testTotalBathRooms
'''
YearRemodAdd 重修时间，如果没有重修时间，则使用YearBuilt替代，从这两个变量可以找出一个新变量，即IsNew变量
IsNew = YearBuilt==YearRemodAdd？1:0
Age = YrSold - YearRemodAdd
'''
trainSet['IsNew']=trainSet.apply(lambda r:1 if r['YearBuilt']==r['YearRemodAdd'] else 0,axis=1)
testSet['IsNew']=testSet.apply(lambda r:1 if r['YearBuilt']==r['YearRemodAdd'] else 0,axis=1)
print(testSet['IsNew'])
trainSet['Age']=trainSet.apply(lambda r:r['YrSold']-r['YearRemodAdd'],axis=1)
testSet['Age']=testSet.apply(lambda r:r['YrSold']-r['YearRemodAdd'],axis=1)
print(testSet['Age'])
'''
查看这两个变量的相关系数
'''
corr = trainSet['Age'].corr(trainSet['SalePrice'])
if abs(corr)<0.5:
   trainSet.drop(['Age'],axis=1)
   testSet.drop(['Age'],axis=1)

'''
TotalBsmtSF 和GrLivArea 加起来是房屋的总空间，即
TotalSqFeet = TotalBsmtSF+GrLivArea
'''
trainSet['TotalSqFeet']=trainSet['TotalBsmtSF']+trainSet['GrLivArea']
testSet['TotalSqFeet']=testSet['TotalBsmtSF']+testSet['GrLivArea']
corr = trainSet['TotalSqFeet'].corr(trainSet['SalePrice'])
if abs(corr)<0.5:
   trainSet.drop(['TotalSqFeet'],axis=1)
   testSet.drop(['TotalSqFeet'],axis=1)
else:
   sns.regplot(trainSet['TotalSqFeet'],trainSet['SalePrice'])
   plt.show()

'''
和porch门廊面积有关的变量
WoodDeckSF
OpenPorchSF
EnclosedPorch
3SsnPorch
ScreenPorch
除了WoodDeckSF之外，剩下的门廊都是有遮挡的，因此将OpenPorchSF EnclosedPorch 3SsnPorch ScreenPorch的面积加起来
TotalPorchSF = OpenPorchSF +EnclosedPorch +3SsnPorch +ScreenPorch
'''
trainSet['TotalPorchSF']=trainSet['OpenPorchSF']+trainSet['EnclosedPorch']+trainSet['3SsnPorch']+trainSet['ScreenPorch']
testSet['TotalPorchSF']=testSet['OpenPorchSF']+testSet['EnclosedPorch']+testSet['3SsnPorch']+testSet['ScreenPorch']

corr = trainSet['TotalPorchSF'].corr(trainSet['SalePrice'])
if abs(corr)<0.5:
   trainSet.drop(['TotalPorchSF'],axis=1)
   testSet.drop(['TotalPorchSF'],axis=1)
else:
   sns.regplot(trainSet['TotalPorchSF'],trainSet['SalePrice'])
   plt.show()

'''
prepare for model
1、去掉高相关性的特征（两个特征之间高相关性，去掉与SalePrice相关性较低的）
'''
numericCols = eda.numericSet(trainSet).columns.tolist()
print(numericCols)
numericSet = trainSet[numericCols].astype('float')
corr = numericSet.corr()
'''
找出和SalePrice相关性比较强的
'''
rel = set([])
for index,row in corr.iterrows():
       if abs(row['SalePrice'])>0.5:
            rel.add(index)
for index,row in corr.iterrows():
       for col in corr.columns:
              if row[col]>0.5 and index!=col and index in rel and col in rel and row['SalePrice']>0.5:
                     print(index,col,row[col])
'''
从中去掉和SalePrice相关性较弱的特征
'YearRemodAdd', 'GarageYrBlt', 'GarageArea', 'GarageCond', 'TotalBsmtSF', 'TotalRmsAbvGrd', 'BsmtFinSF1'
'''
trainSet = trainSet.drop(['YearRemodAdd', 'GarageYrBlt', 'GarageArea', 'GarageCond', 'TotalBsmtSF', 'TotRmsAbvGrd', 'BsmtFinSF1'],axis=1)
testSet = testSet.drop(['YearRemodAdd', 'GarageYrBlt', 'GarageArea', 'GarageCond', 'TotalBsmtSF', 'TotRmsAbvGrd', 'BsmtFinSF1'],axis=1)

print(trainSet.columns.tolist())
'''
将true Numeric 特征归一化，所谓true Numeric，是指的真正的数值，而不是ordinal
true Numeric：
‘Age’，‘BsmtFinSF2’，‘BsmtFullBath’，‘BsmtHalfBath’，‘FullBath’，‘HalfBath’，‘BsmtUnfSF’，LotFrontage', 'LotArea','YearBuilt',MasVnrArea',
'1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea','Fireplaces','GarageCars','WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 
'ScreenPorch', 'PoolArea','MiscVal', 'MoSold', 'YrSold','TotalBathRooms','TotalSqFeet', 'TotalPorchSF','SalePrice'
ordinal:
'Id', 'MSSubClass','MSZoning','Street','Alley','LotShape','LandContour','Utilities','LotConfig', 'LandSlope', 'Neighborhood',
Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'OverallQual', 'OverallCond','RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd',
 'MasVnrType', 'ExterQual', 'ExterCond','Foundation', 'BsmtQual', 'BsmtCond', 'BsmtExposure','BsmtFinType1', 'BsmtFinType2','Heating',
 'HeatingQC', 'CentralAir','Electrical','BedroomAbvGr', 'KitchenAbvGr', 'KitchenQual','Functional','FireplaceQu','GarageType','GarageFinish',
'GarageQual','PavedDrive','PoolQC','Fence','MiscFeature','SaleType', 'SaleCondition','IsNew'
'''
trueNumericCols = ['Age','BsmtFinSF2','BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BsmtUnfSF','LotFrontage', 'LotArea','YearBuilt','MasVnrArea',
'1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea','Fireplaces','GarageCars','WoodDeckSF', 'OpenPorchSF', 'EnclosedPorch', '3SsnPorch', 
'ScreenPorch', 'PoolArea','MiscVal', 'MoSold', 'YrSold','TotalBathRooms','TotalSqFeet', 'TotalPorchSF']
trueCatCols=['MSSubClass','MSZoning','Street','Alley','LotShape','LandContour','Utilities','LotConfig', 'LandSlope', 'Neighborhood',
'Condition1', 'Condition2', 'BldgType', 'HouseStyle', 'OverallQual', 'OverallCond','RoofStyle', 'RoofMatl', 'Exterior1st', 'Exterior2nd',
 'MasVnrType', 'ExterQual', 'ExterCond','Foundation', 'BsmtQual', 'BsmtCond', 'BsmtExposure','BsmtFinType1', 'BsmtFinType2','Heating',
 'HeatingQC', 'CentralAir','Electrical','BedroomAbvGr', 'KitchenAbvGr', 'KitchenQual','Functional','FireplaceQu','GarageType','GarageFinish',
'GarageQual','PavedDrive','PoolQC','Fence','MiscFeature','SaleType', 'SaleCondition','IsNew']
'''
使用峰度Skewness 将数值变量过度不对称的用log压缩
'''
for col in trueNumericCols:
   if abs(trainSet[col].skew())>0.8:
      trainSet[col]=trainSet[col].transform(lambda x:log(x+1))
      testSet[col]=testSet[col].transform(lambda x:log(x+1))


'''
对Numeric特征处理相应的Normalize
'''
for col in trueNumericCols:
      trainSet[col] = preprocessing.minmax_scale(trainSet[col],copy=False)
      testSet[col] = preprocessing.minmax_scale(testSet[col],copy=False)

'''
对Cat特征进行One-hot encoding编码
在trueCatCols中，又分为
Ordinal：'PoolQC','FireplaceQu','GarageCond','GarageFinish','GarageQual','BsmtQual','BsmtCond','BsmtExposure',
         'BsmtFinType1','BsmtFinType2','KitchenQual','Utilities','Functional','ExterQual','ExterCond','HeatingQC',
         'LandSlope','PavedDrive','OverallQual','OverallCond'
Cat：'MSSubClass','MSZoning','Street','Alley','LotShape','LandContour',LotConfig', 'Neighborhood',
     'Condition1', 'Condition2', 'BldgType', 'HouseStyle',  'RoofStyle', 
     'RoofMatl', 'Exterior1st', 'Exterior2nd','MasVnrType','Foundation','Heating','CentralAir','Electrical',
     'BedroomAbvGr', 'KitchenAbvGr', 'GarageType','Fence','MiscFeature','SaleType', 'SaleCondition','IsNew'
'''
catCols = ['MSSubClass','MSZoning','Street','Alley','LotShape','LandContour','LotConfig', 'Neighborhood',
     'Condition1', 'Condition2', 'BldgType', 'HouseStyle',  'RoofStyle', 
     'RoofMatl', 'Exterior1st', 'Exterior2nd','MasVnrType','Foundation','Heating','CentralAir','Electrical',
     'BedroomAbvGr', 'KitchenAbvGr', 'GarageType','Fence','MiscFeature','SaleType', 'SaleCondition','IsNew']
oneHotDf = pd.get_dummies(trainSet[catCols])
testOneHotDf = pd.get_dummies(testSet[catCols])
'''
在两个onehot中，可能存在col不一致的情况，有些训练集合中有的取值测试集中可能没有，需要规范
'''
trainOneHotCols = oneHotDf.columns.tolist()
testOneHotCols = testOneHotDf.columns.tolist()
for col in trainOneHotCols:
       if not (col in testOneHotCols):
              oneHotDf = oneHotDf.drop([col],axis=1)
for col in testOneHotCols:
   if not  (col in trainOneHotCols):
         testOneHotDf=testOneHotDf.drop([col],axsi=1)         
trainSet = trainSet.drop(catCols,axis=1)
testSet = testSet.drop(catCols,axis=1)
print(len(oneHotDf.columns.tolist()),len(trainSet.columns.tolist()))
for col in oneHotDf.columns.tolist():
   '''
   oneHotDf有些col中，为‘1’的值小于10个，几乎等于0，去掉这些列，归于dummy中的‘其他’
   '''
   oneHotCol = oneHotDf[col] 
   if oneHotCol.where(oneHotCol==1).count() >10:
          trainSet[col] = oneHotDf[col]
          testSet[col] = testOneHotDf[col]
print(len(trainSet.columns.tolist()))
'''
处理SalePrice，查看它的skew（偏度），以及QQ图查看是否接近正态分布
'''
stats.probplot(trainSet['SalePrice'], dist="norm", plot=plt)
plt.show()
if abs(trainSet['SalePrice'].skew())>0.8:
   trainSet['SalePrice']=trainSet['SalePrice'].transform(lambda x:log(x))
'''
做过处理之后，基本和正态分布拟合
'''   
stats.probplot(trainSet['SalePrice'], dist="norm", plot=plt)
plt.show()



'''
modeling 使用模型进行训练
'''
from sklearn.model_selection import GridSearchCV
'''
加载结果
'''
TYY = pd.read_csv('average.csv')
TYY['SalePrice']=TYY['SalePrice'].transform(lambda x:log(x))
TY=TYY['SalePrice'].values
print(TY)
'''
1、先使用Lasso进行训练
'''
from sklearn.linear_model import Lasso
lasso = Lasso() 
parameters = {'alpha':[0.01, 0.1, 1.0 ,10, 100, 1000]}
clf=GridSearchCV(lasso,parameters,scoring='neg_mean_squared_error')
X = trainSet.drop(['SalePrice'],axis=1).values
Y = trainSet['SalePrice'].values
clf.fit(X,Y)
print(clf.best_params_) #获取训练会的线性函数X参数的权值
print(clf.fit_params) # 训练后模型截距
TX = testSet.drop(['SalePrice'],axis=1).values
print(clf.predict(TX)) #根据输出值进行预测
print(clf.get_params())
print(clf.score(TX,TY))

'''
2、使用xgboost
'''
n_estimators = [50, 100, 200, 400]
max_depth = [2, 4, 6, 8]
learning_rate = [0.0001, 0.001, 0.01, 0.1, 0.2, 0.3]
param_grid = dict(max_depth=max_depth, n_estimators=n_estimators, learning_rate=learning_rate)
xgb_model = XGBRegressor()
fit_params = {"eval_metric": "rmse"}
clf=GridSearchCV(xgb_model, param_grid, verbose=1, fit_params=fit_params, scoring='neg_mean_squared_error')
grid_result = clf.fit(X, Y)
print(clf.score(TX,TY))